function theisiPrepare()

    clc 
    close all
    bands = 224
    %%Helper.loadAllData(bands)  %% Step1：把原始数据导出为mat文件
    
    %% 7月26日的数据
    datapath = 'F:\12份土壤\matlab\data';%% 反射率提取后的 Mat数据文件
    dataSrcInput = 'F:\12份土壤\matlab\input';%% 原始光谱反射率TXT文件
    fileSavePath = 'F:\12份土壤\matlab\output';%% R2，RMSE结果输出文件
    %% 7月28日的数据
    datapath = 'F:\12份土壤\matlab\data\20220728';%% 反射率提取后的 Mat数据文件
    dataSrcInput = 'F:\12份土壤\matlab\input\20220728';%% 原始光谱反射率TXT文件
    fileSavePath = 'F:\12份土壤\matlab\output';%% R2，RMSE结果输出文件
    bandFile = 'F:\12份土壤\matlab\thesis\bands.txt';
    
    args.bands = bands;
    args.removeHead = 0;%去掉前10    
    args.removeHead = 7;%去掉前10
    args.removeTail = 0;
    args.removeTail = 11;%%去掉后10
    args.containMetal = false;
    args.containMetal = true;%%保留金属
    args.dataSrcInput = dataSrcInput;%%Step 1
    
    args.fileSavePath = fileSavePath;
    args.bandFile = bandFile;
    
    isBatch = true
    isBatch = false
    isBatch = true
    
    DoLoadAllData(datapath, args)
end


function DoLoadAllData(datapath, args)
    step1 = 1;      step2 = 2;     step3 = 4;       step4 = 8;
    step5 = 16;     step6 = 32;    step7 = 128;     step8 = 256;
    step9 = 512;    step10 = 1024; step11 = 2048;
    step12 = step11 * 2;    step13 = step12 * 2;    step14 = step13 * 2;
    step15 = step14 * 2;    step16 = step15 * 2;    step17 = step16 * 2;
    step18 = step17 * 2;    step19 = step18 * 2;    step20 = step19 * 2;
    step21 = step20 * 2;    step22 = step21 * 2;    step23 = step22 * 2;
    step24 = step23 * 2;    step25 = step24 * 2;    step26 = step25 * 2;
    step27 = step26 * 2;    step28 = step27 * 2;    step29 = step28 * 2;

    step = 0;
%     step = step + step1;%%Step1：把原始数据导出为mat文件
    step = step + step2;%%Step2：加载数据，去掉前5后15
    step = step + step3;% 画出全光谱
    
    
%     step = step + step4;% 去除PLSRegression    在5种预处理的数据
%     step = step + step5;%%SVR
%     step = step + step6;%%决策树 与随机森林
%     step = step + step7;%%LSBoost
    
%     step = step + step8;%%画出不同元素的多种回归方法
    
%     step = step + step21;%%计算相关性
%     step = step + step22;%%读取所有数据，画出RPD，RMSE，MAPE三个图
%    step = step + step23;%%整理每个元素的 预处理+模型的数据 
% %    step = step + step24;%%读取所有数据，画出RPD，RMSE，MAPE三个图
% %    step = step + step25; %% 绘制特征选择的波段图
% %    step = step + step26; %% 给出波段 nth -->  nm

%     step = step + step27; %%绘制柱状图 2024-10-22
   
   
%     step = step + step9;%%改CR,       
%     step = step + step10;%%SG平滑
%     step = step + step11;%%MSC
%     step = step + step12;%%SNV
%     step = step + step13;%%MA
%     step = step + step14;%%一阶导数
%     step = step + step15;%%二阶导数


%     step = step + step16;%%SPA    
%     step = step + step17;%%fsrftest 
%     step = step + step18;%fsrnca
%     step = step + step19;%sequentialfs
%     step = step + step20;%relieff
%     step = step + step21%cars 竞争自适应重加权采样 competitive adaptive reweighted sampling
      
% % % % % % % % % % % % % % % % % % % % % % % % % % % % % % % % %     
     bands = args.bands;
     fileSavePath = args.fileSavePath;
     if bitand(step, step1)     
        disp('Step 1. ...............');
        isRefl = true
        Helper.loadAllData(args.dataSrcInput, bands, isRefl, fileMatPath)
        
     end
     
     %% 画出全光谱
     if bitand(step, step2)     
        disp('Step 2. ...............');
% %         loadAllData(bands)
        [y_true, xx]  = loadExcelData(datapath);
        corrcoef(y_true')
%         plotmatrix(y_true')
        bandX = load(args.bandFile, '-ascii');
        
        if args.removeHead > 0 || args.removeTail > 0 
            [xx, bandX] = MyRoi.RemoveBands(xx, args.removeHead, args.removeTail, bandX);
        end
        if args.containMetal == true
            y_true = y_true(6:end, : );
        end
% %         Helper.plot12Curve(xx, '12条光谱曲线');

        
        column = 4;
        y_data = repmat(y_true, 1, 2);%%6*24
        x_data =  Helper.extractMatrix(xx, column);%%204*24
        %%后面都要改
%         ThesisHelper.plotPair(x_data);%% 设置removeHead=0， removeTail=0；
%         
%         Helper.plotPair(x_data);
     end    
     
     if bitand(step, step3)     
        disp('Step 3. ...............');

        [y_true, xx]  = loadExcelData(datapath);        

        if args.containMetal == true
            y_true = y_true(6:end, : );
        end
        column = 4;
        y_data = repmat(y_true, 1, 2);%%6*24
        x_data =  Helper.extractMatrix(xx, column);%%204*24
        
        bandX = load(args.bandFile, '-ascii');
        
        ThesisHelper.plot24Curve(x_data, args.removeHead, args.removeTail, bandX); 
        
%         ThesisHelper.plot24Std(x_data, args.removeHead, args.removeTail); 
     end
% % % % % % % % % % % % % % % % % % % % % % % % % % % % % % % % %    c
     %%%%%%%进行CR滤波  与绘图
     bShow = false
%      bShow = true
     if bitand(step, step9)     
        disp('Step 9. CR滤波...............');

        x_data = PreHelper.DoCR(x_data, bShow);%%x_data, 204*24
        
        saveName = 'F:\12份土壤\matlab\thesis\CR.emf';
        ThesisHelper.plot24CurveNoHT(x_data, saveName, bandX)
     end
     %%%%%%%进行SG滤波  与绘图
     if bitand(step, step10)     
        disp('Step 10. SG滤波...............');

        x_data = PreHelper.DoSG(x_data);%%x_data, 204*24
        
        saveName = 'F:\12份土壤\matlab\thesis\SG.emf';
        ThesisHelper.plot24CurveNoHT(x_data, saveName)
     end
     %%%%%%%进行MSC预处理 与绘图
     if bitand(step, step11)     
        disp('Step 11. MSC预处理...............');
        x_data = PreHelper.DoMSC(x_data);%%x_data, 204*24
        
        saveName = 'F:\12份土壤\matlab\thesis\MSC.emf';
        ThesisHelper.plot24CurveNoHT(x_data, saveName)
     end
     %%%%%%%进行SNV预处理 与绘图  
     if bitand(step, step12)     
        disp('Step 12. SNV预处理...............');
        x_data = PreHelper.DoSNV(x_data, bShow);%%x_data, 204*24 
        
        saveName = 'F:\12份土壤\matlab\thesis\SNV.emf';
        ThesisHelper.plot24CurveNoHT(x_data, saveName)
     end
     %%%%%%%进行1阶导数滤波   与绘图  
     if bitand(step, step14)     
        disp('Step 14. 1阶导数滤波...............');
        bShow = true
        bShow = false
        x_data = PreHelper.DoMA(x_data, 3);%%x_data, 204*24
        x_data = PreHelper.DoDiff(x_data, 1, bShow);%%x_data, 204*24  
        
        saveName = 'F:\12份土壤\matlab\thesis\FD.emf';
        ThesisHelper.plot24CurveNoHT(x_data, saveName)
     end 
% % % % % % % % % % % % % % % % % % % % % % % % % % % % %      
     if bitand(step, step4)     
        disp('Step 4. 取出PLSR的各个预处理数据...............');

        dataPath = 'F:\12份土壤\matlab\thesis\'
        dataFile = 'all-pre-PLSR.txt';
        dataIncputFile = [dataPath, dataFile]

        dataSaveFile = 'thesis-all-pre-PLSR.txt';
        dataOutputFile = [dataPath, dataSaveFile]

        ThesisHelper.SaveDataToFormatFile(dataIncputFile, dataOutputFile);     
     end
% % % % % % % % % % % % % % % % % % % % % % % % % % % % %      
     if bitand(step, step5)     
        disp('Step 5. 取出SVR的各个预处理数据...............');

        dataPath = 'F:\12份土壤\matlab\thesis\'
        dataFile = 'all-pre-SVR.txt';
        dataIncputFile = [dataPath, dataFile]

        dataSaveFile = 'thesis-all-pre-SVR.txt';
        dataOutputFile = [dataPath, dataSaveFile]

        ThesisHelper.SaveDataToFormatFile(dataIncputFile, dataOutputFile);     
     end     
% % % % % % % % % % % % % % % % % % % % % % % % % % % % %      
     if bitand(step, step6)     
        disp('Step 6. 取出RF的各个预处理数据...............');

        dataPath = 'F:\12份土壤\matlab\thesis\'
        dataFile = 'all-pre-RF.txt';
        dataIncputFile = [dataPath, dataFile]

        dataSaveFile = 'thesis-all-pre-RF.txt';
        dataOutputFile = [dataPath, dataSaveFile]

        ThesisHelper.SaveDataToFormatFile(dataIncputFile, dataOutputFile);     
     end         
% % % % % % % % % % % % % % % % % % % % % % % % % % % % %      
     if bitand(step, step7)     
        disp('Step 7. 取出LSBoost的各个预处理数据...............');

        dataPath = 'F:\12份土壤\matlab\thesis\'
        dataFile = 'all-pre-LSB.txt';
        dataIncputFile = [dataPath, dataFile]

        dataSaveFile = 'thesis-all-pre-LSB.txt';
        dataOutputFile = [dataPath, dataSaveFile]

        ThesisHelper.SaveDataToFormatFile(dataIncputFile, dataOutputFile);     
     end           
% % % % % % % % % % % % % % % % % % % % % % % % % % % % % % % % % % % % % % %      
% % % % % % % % % % % % % % % % % % % % % % % % % % % % % % % % % % % % % % % 
% % % % % % % % % % % % % % % % % % % % % % % % % % % % % % % % % % % % % % % 
     if bitand(step, step8)     
        disp('Step 8. 画出不同元素的不同回归方法的RPD与RMSE.............');

        dataPath = 'F:\12份土壤\matlab\thesis\'
        dataFile = cell(4, 1);
        dataFile{1} = 'all-pre-PLSR.txt';
        dataFile{2} = 'all-pre-SVR.txt';
        dataFile{3} = 'all-pre-RF.txt';
        dataFile{4} = 'all-pre-LSB.txt';
        
        data = cell(4, 1);
        for i = 1 : length(dataFile)
            dataIncputFile = [dataPath, dataFile{i}];
            
            fileID = fopen(dataIncputFile);
            tmp = textscan(fileID, '%f');
            tmp = reshape(tmp{1}, [36, 4])'; 
            data{i} = tmp;
            fclose(fileID);   
        end
        dd = cell2mat(data);
        
        fileSave = cell(6, 1);
        fileSave{1} = 'fig3_As_1.emf';
        fileSave{2} = 'fig3_Gd_2.emf';
        fileSave{3} = 'fig3_Cr_3.emf';
        fileSave{4} = 'fig3_Cu_4.emf';
        fileSave{5} = 'fig3_Ni_5.emf';
        fileSave{6} = 'fig3_Pb_6.emf';
        
        for i = 1 : length(fileSave)
            fileSave{i} = [dataPath, fileSave{i}];
        end
        ThesisHelper.DrawMetalsFigure(dd, fileSave);
     end        
     
     
     if bitand(step, step22)
         disp('Step 22. 读取所有数据，画出RPD，RMSE，MAPE三个图.............');
         
         dataPath = 'F:\12份土壤\matlab\thesis\'
         dataFile = 'all-data.txt';
         dataInputFile = [dataPath, dataFile];
         data = load(dataInputFile, '-ascii');
         
         fileSave = fullfile(dataPath, 'table-all-data.txt');
         ThesisHelper.DrawMetricFigures(data, fileSave);
%          fileSave = fullfile(dataPath, 'table-all-data2.txt');
%          ThesisHelper.DrawMetricFigures2(data, fileSave);%%增加全光谱保存
     end
     
     if bitand(step, step23)
         disp('Step 23. 整理每个元素的 预处理+模型的数据............');
         
         dataPath = 'F:\12份土壤\matlab\thesis\'
         dataFile = 'all-data.txt';
         dataInputFile = [dataPath, dataFile];
         data = load(dataInputFile, '-ascii');
         
        fileSave = cell(6, 1);
        fileSave{1} = 'table_As_all2.txt';%%增加全光谱1行
        fileSave{2} = 'table_Cd_all2.txt';
        fileSave{3} = 'table_Cr_all2.txt';
        fileSave{4} = 'table_Cu_all2.txt';
        fileSave{5} = 'table_Ni_all2.txt';
        fileSave{6} = 'table_Pb_all2.txt';
        
        for i = 1 : length(fileSave)
            fileSave{i} = [dataPath, fileSave{i}];
        end
         
%          ThesisHelper.DrawMetricTables(data, fileSave);
         ThesisHelper.DrawMetricTables2(data, fileSave);%%增加全光谱
     end
     
     if bitand(step, step27)
         disp('Step 27. 绘制Bar图............');
         
         dataPath = 'F:\12份土壤\matlab\thesis\'
         
         fileLoad = cell(6, 1);
         fileLoad{1} = 'table_As_all2.txt';%%增加全光谱1行
         fileLoad{2} = 'table_Cd_all2.txt';
         fileLoad{3} = 'table_Cr_all2.txt';
         fileLoad{4} = 'table_Cu_all2.txt';
         fileLoad{5} = 'table_Ni_all2.txt';
         fileLoad{6} = 'table_Pb_all2.txt';
        
         fileSave = cell(6,1);
         fileSave{1} = 'fig_As_all';%%增加全光谱1行
         fileSave{2} = 'fig_Cd_all';
         fileSave{3} = 'fig_Cr_all';
         fileSave{4} = 'fig_Cu_all';
         fileSave{5} = 'fig_Ni_all';
         fileSave{6} = 'fig_Pb_all';         
         for i = 1 : length(fileLoad)
             fileSave{i} = [dataPath, fileSave{i}];
             fileLoad{i} = [dataPath, fileLoad{i}];
         end
         
         fileDiff = cell(3,1);
         fileDiff{1} = '_rpd.emf';
         fileDiff{2} = '_rmse.emf';
         fileDiff{3} = '_mape.emf';
         
         isClose = true;
%          isClose = false;%%是否关闭打开窗口
%          ThesisHelper.DrawMetricTables(data, fileSave);
         ThesisHelper.DrawBarFigure(fileLoad, fileSave, fileDiff, isClose);%%增加全光谱
     end
     
     
     if bitand(step, step24)
         disp('Step 24. 组织特征选择的结果数据 要放入表格............');
         %% 文件要先通过 批处理合并后处理
         
         dataPath = 'F:\12份土壤\matlab\thesis\'
         dataFile = '=fea-metals-all.txt';
         dataInputFile = [dataPath, dataFile];
         data = load(dataInputFile, '-ascii');
         fileSave = [dataPath, 'table-all-features.txt'];

      
         
         ThesisHelper.GetFeatureData(data, fileSave);
     end
     
     
     if bitand(step, step25)
         disp('Step 25. 画出特征选择的波段图............');
         %% 文件要先通过 批处理合并后处理
         
         dataPath = 'F:\12份土壤\matlab\thesis\'
         dataFile = 'waves-opt-metals-all.txt';
         dataInputFile = [dataPath, dataFile];
         waves = load(dataInputFile, '-ascii');
         
        

%          offStart
         setting = [2, 4, 4;
             1, 1, 1;
             1, 3, 1;
             2, 3, 3;
             2, 3, 1;
             1, 3, 1];
         
         for metal = 1 : size(setting, 1)
            preType   = setting(metal, 1);
            feaType   = setting(metal ,3);
             
            X = PreHelper.DoOnePreProcessing(x_data, preType, false);
            
            www = waves(metal, :);
            if metal == 1
               www = [www, 192];%%%针对AS 的 MSC+CARS，要增加一个波段来画
            else
                
            end
            fig = Helper.DrawSelectionOnMeanSpectral2(X', www, 0, 0, bandX)
            

            saveName = [dataPath, 'metal-selection-', num2str(metal), '.emf'];
            f = ThesisHelper.GetAFrame();
            im = frame2im(f);
            saveas(gca, saveName)
         end
         
         
     end
     
     if bitand(step, step26)
         disp('Step 26. 给出具体波段............');
         %% 文件要先通过 批处理合并后处理
         
         dataPath = 'F:\12份土壤\matlab\thesis\'
         dataFile = 'waves-opt-metals-all.txt';
         dataInputFile = [dataPath, dataFile];
         waves = load(dataInputFile, '-ascii');
        
         dataPath2 = 'F:\12份土壤\matlab\input\'
         dataFile = 'bands.txt';
         dataInputFile = [dataPath2, dataFile];
         bands = load(dataInputFile, '-ascii');
         
         
         removeHead = args.removeHead;%去掉前10   
         removeTail = args.removeTail;%%去掉后10
         bands = bands(removeHead+1 : end - removeTail);
         
         data = round(bands(waves));
         dataOutputFile = [dataPath, 'band-data.txt'];
         save(dataOutputFile, '-ascii', '-tabs', 'data')
         
     end
     
     if bitand(step, step21)
         disp('Step 21. 计算波段与数据的相关性............');
         %% 文件要先通过 批处理合并后处理
          dataPath2 = 'F:\12份土壤\matlab\input\'
         dataFile = 'bands.txt';
         dataInputFile = [dataPath2, dataFile];
         bands = load(dataInputFile, '-ascii');         
         
         removeHead = args.removeHead;%去掉前10   
         removeTail = args.removeTail;%%去掉后10
         bands = bands(removeHead+1 : end - removeTail);

% % %         x_data
% % %         y_data
        A = [x_data', y_data'];
        B = corrcoef(A);
        C = B(207:end, 1:206);
%         plot(1:206, C')
        plot(bands, C', 'linewidth', 1.5)

        aa = floor(bands(1)) -1;   bb = ceil(bands(end))+1;
        xlim([aa, bb])
        lgd = legend('As', 'Cd', 'Cr', 'Cu', 'Ni', 'Pb')
        set(lgd,'Position',[0.143557522108218 0.739126983525261 0.231712964265435 0.0869047637212843],...
                      'NumColumns',3);
% %         lgd.NumColumns = 3;
        xlabel('Wavelenths (nm)')
        ylabel('Correlation cooefficients')
     end

% % % % % % % % % % % % % % % % % % % % % % % % % % % 

     
     disp('All done');
end

% % % % % % % % % % % % % % % % % % % % % % % % % % % % % % % % % % % % % 
% % % % % % % % % % % % % % % % % % % % % % % % % % % % % % % % % % % % % % 
% % % % % % % % % % % % % % % % % % % % % % % % % % % % % % 
% % Step2:  加载化学检测数据，加载mat波谱数据
function [y_true, xx] = loadExcelData(datapath)
    fileName = 'data.txt';
    fileFullName = fullfile(datapath, fileName);
    
    fid = fopen(fileFullName);
    lines = textscan(fid,'%s %s %s %f %s %s', 'Delimiter','\t');
    data = lines{4};
    len = length(data);
    y_true = reshape(data, len/12,12);   
    fclose(fid)
    
    imdjson = imageDatastore(datapath, 'FileExtensions',{'.mat'}, 'IncludeSubfolders', false);
    xx = cell(length(imdjson.Files), 1);
    for i = 1 : length(imdjson.Files)  
        fileName = imdjson.Files{i};
        
        [~, fName, ~] = fileparts(fileName);
        lines = textscan(fName, '%d_%d');
        data = load(fileName);
        roi = data.roi;
        stats = roi.stats;
        
        first = lines{1};
        second = lines{2};
%         kk = (first - 1) * 2 + 1 + second - 1;
        kk = (second - 1) * 12 + 1 + first - 1;
        xx{kk} = stats;
    end
end

% % Step1：把原始数据导出为mat文件
function loadAllData(datapath, bands, isRefl)
    
%     fileName = 'F:\12份土壤\hou\1-6_up.txt'
    fileName = fullfile(datapath, '1-6_up.txt')
    roie = MyRoi.LoadBatchRoies(fileName, bands, isRefl);
    
    fileRoiPath  = 'F:\12份土壤\matlab\data';
    roiRange = 1:6;
    suffix = '_1';    
    MyRoi.SaveAllRoi(roie, fileRoiPath, roiRange, suffix)
    disp('done1')
    
%     fileName = 'F:\12份土壤\hou\1-6_down.txt'
    fileName = fullfile(datapath, '1-6_down.txt')
    roie = MyRoi.LoadBatchRoies(fileName, bands, isRefl);
    roiRange = 1:6;
    suffix = '_2';    
    MyRoi.SaveAllRoi(roie, fileRoiPath, roiRange, suffix)
    disp('done2')
    
%     fileName = 'F:\12份土壤\hou\7-12_up.txt'
    fileName = fullfile(datapath, '7-12_up.txt')
    roie = MyRoi.LoadBatchRoies(fileName, bands, isRefl);
    
    roiRange = 7:12;
    suffix = '_1';    
    MyRoi.SaveAllRoi(roie, fileRoiPath, roiRange, suffix)
    disp('done3')
    
%     fileName = 'F:\12份土壤\hou\7-12_down.txt'
    fileName = fullfile(datapath, '7-12_down.txt')
    roie = MyRoi.LoadBatchRoies(fileName, bands, isRefl);   
    
    roiRange = 7:12;
    suffix = '_2';    
    MyRoi.SaveAllRoi(roie, fileRoiPath, roiRange, suffix)
    disp('done4')
end

